# -*- coding: utf-8 -*-
import scrapy
import time
import random

from chengdu.items import ChengduItem


class LianjiaSpider(scrapy.Spider):
    name = 'lianjia'
    allowed_domains = ['cd.lianjia.com']
    base_url = "https://cd.lianjia.com/ershoufang/"
    page_number = 1
    start_urls = [base_url + "pg" + str(page_number) + "/"]

    def parse(self, response):
        li_list = response.xpath('//ul[@class="sellListContent"]/li')
        for li in li_list:
            detail_url = li.xpath('./div[@class="info clear"]/div[@class="title"]/a[1]/@href').get()
            #time.sleep(random.randint(1, 10))
            yield scrapy.Request(detail_url, callback=self.parse_detail)

        if self.page_number < 101:
            self.page_number += 1
            #time.sleep(random.randint(1,10))
            yield scrapy.Request(self.base_url + "pg" + str(self.page_number) + "/", callback=self.parse)

    def parse_detail(self, response):
        item = ChengduItem()
        title = response.xpath('//div[@class="sellDetailHeader"][1]/div[@class="title-wrapper"]/div[@class="content"]/div[@class="title"]/h1[1]/text()').get()
        item['title'] = title
        overview_content = response.xpath('//div[@class="overview"]/div[@class="content"]')
        region = overview_content.xpath('./div[@class="aroundInfo"]/div[@class="areaName"]/span[@class="info"]/a[1]/text()').get()
        item['region'] = region
        address = overview_content.xpath('./div[@class="aroundInfo"]/div[@class="areaName"]/span[@class="info"]/a/text()').getall()
        item['address'] = "-".join(address)
        housing_estate = overview_content.xpath('./div[@class="aroundInfo"]/div[@class="communityName"]/a[1]/text()').get()
        item['housing_estate'] = housing_estate
        total_price = overview_content.xpath('./div[@class="price "]/span[@class="total"][1]/text()').get()
        item['total_price'] = total_price
        unit_price = overview_content.xpath('./div[@class="price "]/div[@class="text"]/div[@class="unitPrice"]/span[@class="unitPriceValue"]/text()').get()
        item['unit_price'] = unit_price
        room_type = overview_content.xpath('./div[@class="houseInfo"]/div[@class="room"]/div[@class="mainInfo"]/text()').get()
        item['room_type'] = room_type
        room_size = overview_content.xpath('./div[@class="houseInfo"]/div[@class="area"]/div[@class="mainInfo"]/text()').get()
        item['room_size'] = room_size
        floor = overview_content.xpath('./div[@class="houseInfo"]/div[@class="room"]/div[@class="subInfo"]/text()').get()
        item['floor'] = floor
        direction = overview_content.xpath('./div[@class="houseInfo"]/div[@class="type"]/div[@class="mainInfo"]/text()').get()
        item['direction'] = direction
        house_type = overview_content.xpath('./div[@class="houseInfo"]/div[@class="type"]/div[@class="subInfo"]/text()').get()
        item['house_type'] = house_type
        created_date = overview_content.xpath('./div[@class="houseInfo"]/div[@class="area"]/div[@class="subInfo"]/text()').get()
        item['created_date'] = created_date

        house_info = ""
        lis = response.xpath('//div[@class="m-content"]/div[@class="box-l"]/div[@class="newwrap baseinform"]/div[1]/div[@class="introContent"]/div[@class="base"]/div[@class="content"]/ul/li')
        for li in lis:
            field_name = li.xpath('./span/text()').get()
            field_value = li.xpath('./text()').get()
            house_info = house_info + field_name + ":" + field_value + ","
        item['house_info'] = house_info

        transactions = ""
        hang_out_date = ""
        lis = response.xpath('//div[@class="m-content"]/div[@class="box-l"]/div[@class="newwrap baseinform"]/div[1]/div[@class="introContent"]/div[@class="transaction"]/div[@class="content"]/ul/li')
        index = 0
        for li in lis:
            field_name = li.xpath('./span[1]/text()').get()
            field_value = li.xpath('./span[2]/text()').get()
            if index == 0:
                hang_out_date = field_value
            transactions = transactions + field_name + ":" + field_value + ","
            index += 1
        item['hang_out_date'] = hang_out_date
        transactions = transactions.replace(" ","")
        transactions = transactions.replace("\n", "")
        #print(transactions)
        item['transactions'] = transactions
        yield item